import { TextLoader } from "langchain/document_loaders/fs/text";
import { RecursiveCharacterTextSplitter } from "langchain/text_splitter";

const loader = new TextLoader("data/kong.txt");

const docs = await loader.load();

const splitter = new RecursiveCharacterTextSplitter({
  chunkSize: 300,
  chunkOverlap: 0,
});

const splittedDocs = await splitter.splitDocuments(docs);

// console.log(splittedDocs)

let active = 0; // 嵌入操作的计数器
async function getEmbedding(text) {
  active++;
  console.log("active+++>", active);
  const res = await fetch("http://localhost:11434/api/embeddings", {
    method: "POST",
    headers: { "Content-Type": "application/json" },
    body: JSON.stringify({
      model: "nomic-embed-text",
      prompt: text,
    }),
  });
  const result = await res.json();
  active--;
  console.log("active--->", active);
  return result.embedding;
}

// 开始计时
const startTime = performance.now();

const results = [];
for (const doc of splittedDocs) {
  const embedding = await getEmbedding(doc.pageContent);
  results.push({ ...doc, embedding });
}

console.log(results);

// 计时结束
const endTime = performance.now();
console.log(`总耗时：${endTime - startTime} 毫秒`);
